/*
 * Copyright (C) 2010  Paulo Cesar Pereira de Andrade.
 *
 * This is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This software is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * Authors:
 *	Paulo Cesar Pereira de Andrade
 */

#if PROTO
#define icode				0
#define fcode				1
#define acode				2

static elit_node_t *
node_add(int of, elit_node_t *lit);

static void
node_link(elit_node_t *label, elit_node_t *jump);

#define new_calli(of, i0)		new_p(of, elit_calli,i0)
#define new_finish(of, i0)		new_p(of, elit_finish, i0)
static elit_node_t *
new_p(int of, elit_code_t c, void *i0);

static elit_node_t *
new_label(int of, elit_node_t *jump);

static elit_node_t *
new_jmpi(int of, elit_node_t *label);

#define new_prepare(of, i0)		new_i(of, elit_prepare, i0)
static elit_node_t *
new_i(int o, elit_code_t c, int i0);

#define new_pusharg_i(of, r0)		new_r(of, elit_pusharg_i, r0)
#define new_pusharg_l(of, r0)		new_r(of, elit_pusharg_l, r0)
#define new_pusharg_p(of, r0)		new_r(of, elit_pusharg_p, r0)
static elit_node_t *
new_r(int of, elit_code_t c, int r0);

#define new_movi_i(of, r0, i0)		new_ri(of, elit_movi_i, r0, i0)
static elit_node_t *
new_ri(int of, elit_code_t c, int r0, int i0);

#define new_movi_l(of, r0, i0)		new_rl(of, elit_movi_l, r0, i0)
static elit_node_t *
new_rl(int of, elit_code_t c, int r0, long i0) maybe_unused;

#define new_movi_p(of, r0, i0)		new_rp(of, elit_movi_p, r0, i0)
#define new_ldi_p(of, r0, i0)		new_rp(of, elit_ldi_p, r0, i0)
static elit_node_t *
new_rp(int of, elit_code_t c, int r0, void *i0);

#define new_movi_f(of, r0, i0)		new_rf(of, elit_movi_f, r0, i0)
static elit_node_t *
new_rf(int of, elit_code_t c, int r0, float i0) maybe_unused;

#define new_movi_d(of, r0, i0)		new_rd(of, elit_movi_d, r0, i0)
static elit_node_t *
new_rd(int of, elit_code_t c, int r0, double i0);

#define new_movr_i(of, r0, r1)		new_rr(of, elit_movr_i, r0, r1)
#define new_movr_l(of, r0, r1)		new_rr(of, elit_movr_l, r0, r1)
#define new_notr_i(of, r0, r1)		new_rr(of, elit_notr_i, r0, r1)
#define new_notr_l(of, r0, r1)		new_rr(of, elit_notr_l, r0, r1)
#define new_negr_i(of, r0, r1)		new_rr(of, elit_negr_i, r0, r1)
#define new_negr_l(of, r0, r1)		new_rr(of, elit_negr_l, r0, r1)
#define new_negr_f(of, r0, r1)		new_rr(of, elit_negr_f, r0, r1)
#define new_negr_d(of, r0, r1)		new_rr(of, elit_negr_d, r0, r1)
#define new_absr_f(of, r0, r1)		new_rr(of, elit_absr_f, r0, r1)
#define new_absr_d(of, r0, r1)		new_rr(of, elit_absr_d, r0, r1)
#define new_extr_i_f(of, r0, r1)	new_rr(of, elit_extr_i_f, r0, r1)
#define new_extr_i_d(of, r0, r1)	new_rr(of, elit_extr_i_d, r0, r1)
#define new_extr_l_f(of, r0, r1)	new_rr(of, elit_extr_l_f, r0, r1)
#define new_extr_l_d(of, r0, r1)	new_rr(of, elit_extr_l_d, r0, r1)
#define new_extr_f_d(of, r0, r1)	new_rr(of, elit_extr_f_d, r0, r1)
#define new_extr_d_f(of, r0, r1)	new_rr(of, elit_extr_d_f, r0, r1)
#define new_floorr_f_i(of, r0, r1)	new_rr(of, elit_floorr_f_i, r0, r1)
#define new_floorr_f_l(of, r0, r1)	new_rr(of, elit_floorr_f_l, r0, r1)
#define new_floorr_d_i(of, r0, r1)	new_rr(of, elit_floorr_d_i, r0, r1)
#define new_floorr_d_l(of, r0, r1)	new_rr(of, elit_floorr_d_l, r0, r1)
#define new_truncr_f_i(of, r0, r1)	new_rr(of, elit_truncr_f_i, r0, r1)
#define new_truncr_f_l(of, r0, r1)	new_rr(of, elit_truncr_f_l, r0, r1)
#define new_truncr_d_i(of, r0, r1)	new_rr(of, elit_truncr_d_i, r0, r1)
#define new_truncr_d_l(of, r0, r1)	new_rr(of, elit_truncr_d_l, r0, r1)
static elit_node_t *
new_rr(int o, elit_code_t c, int r0, int r1);

#define new_blti_i(of, i0, r0, i1)	new_pri(of, elit_blti_i, i0, r0, i1)
#define new_blei_i(of, i0, r0, i1)	new_pri(of, elit_blei_i, i0, r0, i1)
#define new_beqi_i(of, i0, r0, i1)	new_pri(of, elit_beqi_i, i0, r0, i1)
#define new_bgei_i(of, i0, r0, i1)	new_pri(of, elit_bgei_i, i0, r0, i1)
#define new_bgti_i(of, i0, r0, i1)	new_pri(of, elit_bgti_i, i0, r0, i1)
#define new_bnei_i(of, i0, r0, i1)	new_pri(of, elit_bnei_i, i0, r0, i1)
#define new_boaddi_i(of, i0, r0, i1)	new_pri(of, elit_boaddi_i, i0, r0, i1)
#define new_boaddi_ui(of, i0, r0, i1)	new_pri(of, elit_boaddi_ui, i0, r0, i1)
#define new_bosubi_i(of, i0, r0, i1)	new_pri(of, elit_bosubi_i, i0, r0, i1)
#define new_bosubi_ui(of, i0, r0, i1)	new_pri(of, elit_bosubi_ui, i0, r0, i1)
static elit_node_t *
new_pri(int of, elit_code_t c, elit_node_t *i0, int r0, int i1);

#define new_blti_l(of, i0, r0, i1)	new_prl(of, elit_blti_l, i0, r0, i1)
#define new_blei_l(of, i0, r0, i1)	new_prl(of, elit_blei_l, i0, r0, i1)
#define new_beqi_l(of, i0, r0, i1)	new_prl(of, elit_beqi_l, i0, r0, i1)
#define new_bgei_l(of, i0, r0, i1)	new_prl(of, elit_bgei_l, i0, r0, i1)
#define new_bgti_l(of, i0, r0, i1)	new_prl(of, elit_bgti_l, i0, r0, i1)
#define new_bnei_l(of, i0, r0, i1)	new_prl(of, elit_bnei_l, i0, r0, i1)
#define new_boaddi_l(of, i0, r0, i1)	new_prl(of, elit_boaddi_l, i0, r0, i1)
#define new_boaddi_ul(of, i0, r0, i1)	new_prl(of, elit_boaddi_ul, i0, r0, i1)
#define new_bosubi_l(of, i0, r0, i1)	new_prl(of, elit_bosubi_l, i0, r0, i1)
static elit_node_t *
new_prl(int of, elit_code_t c, elit_node_t *i0, int r0, long i1) maybe_unused;

#define new_boaddr_i(of, i0, r0, r1)	new_prr(of, elit_boaddr_i, i0, r0, r1)
#define new_boaddr_l(of, i0, r0, r1)	new_prr(of, elit_boaddr_l, i0, r0, r1)
#define new_bosubr_i(of, i0, r0, r1)	new_prr(of, elit_bosubr_i, i0, r0, r1)
#define new_bosubr_l(of, i0, r0, r1)	new_prr(of, elit_bosubr_l, i0, r0, r1)
#define new_bger_f(of, i0, r0, r1)	new_prr(of, elit_bger_f, i0, r0, r1)
#define new_bger_d(of, i0, r0, r1)	new_prr(of, elit_bger_d, i0, r0, r1)
static elit_node_t *
new_prr(int of, elit_code_t c, elit_node_t *i0, int r0, int r1) maybe_unused;

#define new_addi_i(of, r0, r1, i0)	new_rri(of, elit_addi_i, r0, r1, i0)
#define new_addi_p(of, r0, r1, i0)	new_rri(of, elit_addi_p, r0, r1, i0)
#define new_addci_ui(of, r0, r1, i0)	new_rri(of, elit_addci_ui, r0, r1, i0)
#define new_addxi_ui(of, r0, r1, i0)	new_rri(of, elit_addxi_ui, r0, r1, i0)
#define new_subi_i(of, r0, r1, i0)	new_rri(of, elit_subi_i, r0, r1, i0)
#define new_subi_p(of, r0, r1, i0)	new_rri(of, elit_subi_p, r0, r1, i0)
#define new_subci_ui(of, r0, r1, i0)	new_rri(of, elit_subci_ui, r0, r1, i0)
#define new_subxi_ui(of, r0, r1, i0)	new_rri(of, elit_subxi_ui, r0, r1, i0)
#define new_rshi_i(of, r0, r1, i0)	new_rri(of, elit_rshi_i, r0, r1, i0)
#define new_nei_i(of, r0, r1, i0)	new_rri(of, elit_nei_i, r0, r1, i0)
#define new_lti_i(of, r0, r1, i0)	new_rri(of, elit_lti_i, r0, r1, i0)
#define new_lei_i(of, r0, r1, i0)	new_rri(of, elit_lei_i, r0, r1, i0)
#define new_eqi_i(of, r0, r1, i0)	new_rri(of, elit_eqi_i, r0, r1, i0)
#define new_gei_i(of, r0, r1, i0)	new_rri(of, elit_gei_i, r0, r1, i0)
#define new_gti_i(of, r0, r1, i0)	new_rri(of, elit_gti_i, r0, r1, i0)
#define new_ldxi_c(of, r0, r1, i0)	new_rri(of, elit_ldxi_c, r0, r1, i0)
#define new_ldxi_uc(of, r0, r1, i0)	new_rri(of, elit_ldxi_uc, r0, r1, i0)
#define new_ldxi_s(of, r0, r1, i0)	new_rri(of, elit_ldxi_s, r0, r1, i0)
#define new_ldxi_us(of, r0, r1, i0)	new_rri(of, elit_ldxi_us, r0, r1, i0)
#define new_ldxi_i(of, r0, r1, i0)	new_rri(of, elit_ldxi_i, r0, r1, i0)
#define new_ldxi_ui(of, r0, r1, i0)	new_rri(of, elit_ldxi_ui, r0, r1, i0)
#define new_ldxi_l(of, r0, r1, i0)	new_rri(of, elit_ldxi_l, r0, r1, i0)
#define new_ldxi_ul(of, r0, r1, i0)	new_rri(of, elit_ldxi_ul, r0, r1, i0)
#define new_ldxi_p(of, r0, r1, i0)	new_rri(of, elit_ldxi_p, r0, r1, i0)
#define new_ldxi_f(of, r0, r1, i0)	new_rri(of, elit_ldxi_f, r0, r1, i0)
#define new_ldxi_d(of, r0, r1, i0)	new_rri(of, elit_ldxi_d, r0, r1, i0)
#define new_stxi_c(of, i0, r0, r1)	new_rri(of, elit_stxi_c, r0, r1, i0)
#define new_stxi_s(of, i0, r0, r1)	new_rri(of, elit_stxi_s, r0, r1, i0)
#define new_stxi_i(of, i0, r0, r1)	new_rri(of, elit_stxi_i, r0, r1, i0)
#define new_stxi_l(of, i0, r0, r1)	new_rri(of, elit_stxi_l, r0, r1, i0)
#define new_stxi_p(of, i0, r0, r1)	new_rri(of, elit_stxi_p, r0, r1, i0)
#define new_stxi_f(of, i0, r0, r1)	new_rri(of, elit_stxi_f, r0, r1, i0)
#define new_stxi_d(of, i0, r0, r1)	new_rri(of, elit_stxi_d, r0, r1, i0)
static elit_node_t *
new_rri(int of, elit_code_t c, int r0, int r1, int i0);

#define new_addi_l(of, r0, r1, i0)	new_rrl(of, elit_addi_l, r0, r1, i0)
#define new_subi_l(of, r0, r1, i0)	new_rrl(of, elit_subi_l, r0, r1, i0)
#define new_nei_l(of, r0, r1, i0)	new_rrl(of, elit_nei_i, r0, r1, i0)
#define new_lti_l(of, r0, r1, i0)	new_rrl(of, elit_lti_i, r0, r1, i0)
#define new_lei_l(of, r0, r1, i0)	new_rrl(of, elit_lei_i, r0, r1, i0)
#define new_eqi_l(of, r0, r1, i0)	new_rrl(of, elit_eqi_i, r0, r1, i0)
#define new_gei_l(of, r0, r1, i0)	new_rrl(of, elit_gei_i, r0, r1, i0)
#define new_gti_l(of, r0, r1, i0)	new_rrl(of, elit_gti_i, r0, r1, i0)
static elit_node_t *
new_rrl(int of, elit_code_t c, int r0, int r1, long i0) maybe_unused;

#define new_addr_i(of, r0, r1, r2)	new_rrr(of, elit_addr_i, r0, r1, r2)
#define new_addr_ui(of, r0, r1, r2)	new_rrr(of, elit_addr_ui, r0, r1, r2)
#define new_addr_l(of, r0, r1, r2)	new_rrr(of, elit_addr_l, r0, r1, r2)
#define new_addr_f(of, r0, r1, r2)	new_rrr(of, elit_addr_f, r0, r1, r2)
#define new_addr_d(of, r0, r1, r2)	new_rrr(of, elit_addr_d, r0, r1, r2)
#define new_addcr_ui(of, r0, r1, r2)	new_rrr(of, elit_addcr_ui, r0, r1, r2)
#define new_addxr_ui(of, r0, r1, r2)	new_rrr(of, elit_addxr_ui, r0, r1, r2)
#define new_subr_i(of, r0, r1, r2)	new_rrr(of, elit_subr_i, r0, r1, r2)
#define new_subr_ui(of, r0, r1, r2)	new_rrr(of, elit_subr_ui, r0, r1, r2)
#define new_subr_l(of, r0, r1, r2)	new_rrr(of, elit_subr_l, r0, r1, r2)
#define new_subr_f(of, r0, r1, r2)	new_rrr(of, elit_subr_f, r0, r1, r2)
#define new_subr_d(of, r0, r1, r2)	new_rrr(of, elit_subr_d, r0, r1, r2)
#define new_subcr_ui(of, r0, r1, r2)	new_rrr(of, elit_subcr_ui, r0, r1, r2)
#define new_subxr_ui(of, r0, r1, r2)	new_rrr(of, elit_subxr_ui, r0, r1, r2)
#define new_mulr_f(of, r0, r1, r2)	new_rrr(of, elit_mulr_f, r0, r1, r2)
#define new_mulr_d(of, r0, r1, r2)	new_rrr(of, elit_mulr_d, r0, r1, r2)
#define new_divr_f(of, r0, r1, r2)	new_rrr(of, elit_divr_f, r0, r1, r2)
#define new_divr_d(of, r0, r1, r2)	new_rrr(of, elit_divr_d, r0, r1, r2)
#define new_ner_i(of, r0, r1, r2)	new_rrr(of, elit_ner_i, r0, r1, r2)
#define new_ner_l(of, r0, r1, r2)	new_rrr(of, elit_ner_l, r0, r1, r2)
#define new_ner_d(of, r0, r1, r2)	new_rrr(of, elit_ner_d, r0, r1, r2)
#define new_ltr_i(of, r0, r1, r2)	new_rrr(of, elit_ltr_i, r0, r1, r2)
#define new_ltr_ui(of, r0, r1, r2)	new_rrr(of, elit_ltr_ui, r0, r1, r2)
#define new_ltr_l(of, r0, r1, r2)	new_rrr(of, elit_ltr_l, r0, r1, r2)
#define new_ltr_d(of, r0, r1, r2)	new_rrr(of, elit_ltr_d, r0, r1, r2)
#define new_ler_i(of, r0, r1, r2)	new_rrr(of, elit_ler_i, r0, r1, r2)
#define new_ler_ui(of, r0, r1, r2)	new_rrr(of, elit_ler_ui, r0, r1, r2)
#define new_ler_l(of, r0, r1, r2)	new_rrr(of, elit_ler_l, r0, r1, r2)
#define new_ler_d(of, r0, r1, r2)	new_rrr(of, elit_ler_d, r0, r1, r2)
#define new_eqr_i(of, r0, r1, r2)	new_rrr(of, elit_eqr_i, r0, r1, r2)
#define new_eqr_l(of, r0, r1, r2)	new_rrr(of, elit_eqr_l, r0, r1, r2)
#define new_eqr_d(of, r0, r1, r2)	new_rrr(of, elit_eqr_d, r0, r1, r2)
#define new_ger_i(of, r0, r1, r2)	new_rrr(of, elit_ger_i, r0, r1, r2)
#define new_ger_ui(of, r0, r1, r2)	new_rrr(of, elit_ger_ui, r0, r1, r2)
#define new_ger_l(of, r0, r1, r2)	new_rrr(of, elit_ger_l, r0, r1, r2)
#define new_ger_d(of, r0, r1, r2)	new_rrr(of, elit_ger_d, r0, r1, r2)
#define new_gtr_i(of, r0, r1, r2)	new_rrr(of, elit_gtr_i, r0, r1, r2)
#define new_gtr_ui(of, r0, r1, r2)	new_rrr(of, elit_gtr_ui, r0, r1, r2)
#define new_gtr_l(of, r0, r1, r2)	new_rrr(of, elit_gtr_l, r0, r1, r2)
#define new_gtr_d(of, r0, r1, r2)	new_rrr(of, elit_gtr_d, r0, r1, r2)
static elit_node_t *
new_rrr(int o, elit_code_t c, int r0, int r1, int r2);

static void
flush_list(elit_node_t *t);

static void
flush_code(void);

static void
push_type(eint32_t type);

static eint32_t
pop_type(void) maybe_unused;

static eint32_t
stack_adjust(eint32_t count);

static eint32_t
top_type(void);

static void
zero_sign_extend(eint32_t code, eint32_t type);

static void
float_extend(eint32_t code, eint32_t type);

static void
load_ireg(eint32_t code, jit_gpr_t base, eint32_t offset);

static void
load_ireg_type(eint32_t code, eint32_t offset);

static void
load_ireg_implicit(eint32_t code);

static void
store_ireg(eint32_t code, jit_gpr_t base, eint32_t offset);

static void
store_ireg_value_type(eint32_t code, jit_gpr_t base, eint32_t offset);

static void
store_ireg_implicit(eint32_t code, ebool_t type);

#if __WORDSIZE == 32
static void
store_ireg_implicit_rgz(eint32_t code, ebool_t sign, jit_gpr_t l, jit_gpr_t h);
#else
static void
store_ireg_implicit_rgz(eint32_t code, ebool_t sign, jit_gpr_t r);
#endif

static void
store_int32(eint32_t code, eint32_t offset, eint32_t type);

static void
store_int(eint32_t code, eint32_t offset, eint32_t type);

static void
push_ireg(eint32_t code);

static void
load_freg(eint32_t code, jit_gpr_t base, eint32_t offset);

static void
load_freg_type(eint32_t code, eint32_t offset);

static void
load_freg_implicit(eint32_t code);

static void
store_freg(eint32_t code, jit_gpr_t base, eint32_t offset);

static void
store_freg_value_type(eint32_t code, jit_gpr_t base, eint32_t offset);

static void
store_freg_implicit(eint32_t code, ebool_t type);

static void
store_float32(eint32_t code, eint32_t offset, eint32_t type);

static void
store_float(eint32_t code, eint32_t offset, eint32_t type);

static void
push_freg(eint32_t code);

static void
store_dynamic(eint32_t code, eint32_t offset, eint32_t type);

static void
call_i(eint32_t code, eobject_t function, eint32_t arg0);

static void
emit_sync_uint(eint32_t code);

static void
parse_flush(eint32_t count);

static void
emit_clear(eint32_t count);

static void
emit_flush(eint32_t count);

static void
emit_sync(eint32_t count);
#endif

#if CODE
static elit_node_t *
node_add(eint32_t ofs, elit_node_t *lit)
{
    if (lit_base[ofs] == null)
	lit_base[ofs] = lit_tail[ofs] = lit;
    else {
	lit_tail[ofs]->next = lit;
	lit_tail[ofs] = lit;
    }
    return (lit);
}

static void
node_link(elit_node_t *label, elit_node_t *jump)
{
    assert(label->code == elit_label);
    jump->link = label->i0.node;
    label->i0.node = jump;
}

static elit_node_t *
new_p(int of, elit_code_t c, void *i0)
{
    elit_node_t		* lit;
    enew(&lit,		  lit_node);
    lit->code		= c;
    lit->i0.p		= i0;
    return (node_add(of,  lit));
}

static elit_node_t *
new_label(int of, elit_node_t *jump)
{
    return (new_p(of, elit_label, jump));
}

static elit_node_t *
new_jmpi(int of, elit_node_t *label)
{
    elit_node_t		* lit;
    lit = new_p(of, elit_jmpi, label);
    if (label)
	node_link(label, lit);
    return (lit);
}

static elit_node_t *
new_i(int of, elit_code_t c, int i0)
{
    elit_node_t		* lit;
    enew(&lit,		  lit_node);
    lit->code		= c;
    lit->i0.i		= i0;
    return (node_add(of,  lit));
}

static elit_node_t *
new_r(int of, elit_code_t c, int r0)
{
    elit_node_t		* lit;
    enew(&lit,		  lit_node);
    lit->code		= c;
    lit->r0		= r0;
    return (node_add(of,  lit));
}

static elit_node_t *
new_ri(int of, elit_code_t c, int r0, int i0)
{
    elit_node_t		* lit;
    enew(&lit,		  lit_node);
    lit->code		= c;
    lit->r0		= r0;
    lit->i0.i		= i0;
    return (node_add(of,  lit));
}

static elit_node_t *
new_rl(int of, elit_code_t c, int r0, long i0)
{
    elit_node_t		* lit;
    enew(&lit,		  lit_node);
    lit->code		= c;
    lit->r0		= r0;
    lit->i0.l		= i0;
    return (node_add(of,  lit));
}

static elit_node_t *
new_rp(int of, elit_code_t c, int r0, void *i0)
{
    elit_node_t		* lit;
    enew(&lit,		  lit_node);
    lit->code		= c;
    lit->r0		= r0;
    lit->i0.p		= i0;
    return (node_add(of,  lit));
}

static elit_node_t *
new_rf(int of, elit_code_t c, int r0, float i0)
{
    elit_node_t		* lit;
    enew(&lit,		  lit_node);
    lit->code		= c;
    lit->r0		= r0;
    lit->i0.f		= i0;
    return (node_add(of,  lit));
}

static elit_node_t *
new_rd(int of, elit_code_t c, int r0, double i0)
{
    elit_node_t		* lit;
    enew(&lit,		  lit_node);
    lit->code		= c;
    lit->r0		= r0;
    lit->i0.d		= i0;
    return (node_add(of,  lit));
}

static elit_node_t *
new_rr(int of, elit_code_t c, int r0, int r1)
{
    elit_node_t		* lit;
    enew(&lit,		  lit_node);
    lit->code		= c;
    lit->r0		= r0;
    lit->r1		= r1;
    return (node_add(of,  lit));
}

static elit_node_t *
new_pri(int of, elit_code_t c, elit_node_t *i0, int r0, int i1)
{
    elit_node_t		* lit;
    enew(&lit,		  lit_node);
    lit->code		= c;
    if (i0)		  node_link(i0, lit);
    lit->r0		= r0;
    lit->i1.i		= i1;
    return (node_add(of,  lit));
}

static elit_node_t *
new_prl(int of, elit_code_t c, elit_node_t *i0, int r0, long i1)
{
    elit_node_t		* lit;
    enew(&lit,		  lit_node);
    lit->code		= c;
    if (i0)		  node_link(i0, lit);
    lit->r0		= r0;
    lit->i1.l		= i1;
    return (node_add(of,  lit));
}

static elit_node_t *
new_prr(int of, elit_code_t c, elit_node_t *i0, int r0, int r1)
{
    elit_node_t		* lit;
    enew(&lit,		  lit_node);
    lit->code		= c;
    if (i0)		  node_link(i0, lit);
    lit->r0		= r0;
    lit->r1		= r1;
    return (node_add(of,  lit));
}

static elit_node_t *
new_rri(int of, elit_code_t c, int r0, int r1, int i0)
{
    elit_node_t		* lit;
    enew(&lit,		  lit_node);
    lit->code		= c;
    lit->r0		= r0;
    lit->r1		= r1;
    lit->i0.i		= i0;
    return (node_add(of,  lit));
}

static elit_node_t *
new_rrl(int of, elit_code_t c, int r0, int r1, long i0)
{
    elit_node_t		* lit;
    enew(&lit,		  lit_node);
    lit->code		= c;
    lit->r0		= r0;
    lit->r1		= r1;
    lit->i0.l		= i0;
    return (node_add(of,  lit));
}

static elit_node_t *
new_rrr(int of, elit_code_t c, int r0, int r1, int r2)
{
    elit_node_t		* lit;
    enew(&lit,		  lit_node);
    lit->code		= c;
    lit->r0		= r0;
    lit->r1		= r1;
    lit->r2		= r2;
    return (node_add(of,  lit));
}

static void
flush_list(elit_node_t *lit)
{
    elit_node_t		*node;

    for (; lit; lit = lit->next) {
	switch (lit->code) {
	    case elit_label:
		/* no forward jumps generated */
		for (node = lit->i0.node; node; node = node->link)
		    jit_patch(node->i0.p);
		break;
	    case elit_movr_i:
		jit_movr_i(lit->r0, lit->r1);
		break;
	    case elit_movr_l:
		jit_movr_l(lit->r0, lit->r1);
		break;
	    case elit_movi_i:
		jit_movi_i(lit->r0, lit->i0.i);
		break;
	    case elit_movi_l:
		jit_movi_l(lit->r0, lit->i0.l);
		break;
	    case elit_movi_p:
		jit_movi_p(lit->r0, lit->i0.p);
		break;
	    case elit_movi_f:
		jit_movi_f(lit->r0, lit->i0.f);
		break;
	    case elit_movi_d:
		jit_movi_d(lit->r0, lit->i0.d);
		break;
	    case elit_notr_i:
		jit_notr_i(lit->r0, lit->r1);
		break;
	    case elit_notr_l:
		jit_notr_l(lit->r0, lit->r1);
		break;
	    case elit_negr_i:
		jit_negr_i(lit->r0, lit->r1);
		break;
	    case elit_negr_l:
		jit_negr_l(lit->r0, lit->r1);
		break;
	    case elit_negr_f:
		jit_negr_f(lit->r0, lit->r1);
		break;
	    case elit_negr_d:
		jit_negr_d(lit->r0, lit->r1);
		break;
	    case elit_absr_f:
		jit_absr_f(lit->r0, lit->r1);
		break;
	    case elit_absr_d:
		jit_absr_d(lit->r0, lit->r1);
		break;
	    case elit_extr_i_f:
		jit_extr_i_f(lit->r0, lit->r1);
		break;
	    case elit_extr_i_d:
		jit_extr_i_d(lit->r0, lit->r1);
		break;
	    case elit_extr_l_f:
		jit_extr_l_f(lit->r0, lit->r1);
		break;
	    case elit_extr_l_d:
		jit_extr_l_d(lit->r0, lit->r1);
		break;
	    case elit_extr_f_d:
		jit_extr_f_d(lit->r0, lit->r1);
		break;
	    case elit_extr_d_f:
		jit_extr_d_f(lit->r0, lit->r1);
		break;
	    case elit_floorr_f_i:
		jit_floorr_f_i(lit->r0, lit->r1);
		break;
	    case elit_floorr_f_l:
		jit_floorr_f_l(lit->r0, lit->r1);
		break;
	    case elit_floorr_d_i:
		jit_floorr_d_i(lit->r0, lit->r1);
		break;
	    case elit_floorr_d_l:
		jit_floorr_d_l(lit->r0, lit->r1);
		break;
	    case elit_truncr_f_i:
		jit_truncr_f_i(lit->r0, lit->r1);
		break;
	    case elit_truncr_f_l:
		jit_truncr_f_l(lit->r0, lit->r1);
		break;
	    case elit_truncr_d_i:
		jit_truncr_d_i(lit->r0, lit->r1);
		break;
	    case elit_truncr_d_l:
		jit_truncr_d_l(lit->r0, lit->r1);
		break;
	    case elit_addr_i:
		jit_addr_i(lit->r0, lit->r1, lit->r2);
		break;
	    case elit_addr_ui:
		jit_addr_ui(lit->r0, lit->r1, lit->r2);
		break;
	    case elit_addr_l:
		jit_addr_l(lit->r0, lit->r1, lit->r2);
		break;
	    case elit_addr_f:
		jit_addr_f(lit->r0, lit->r1, lit->r2);
		break;
	    case elit_addr_d:
		jit_addr_d(lit->r0, lit->r1, lit->r2);
		break;
	    case elit_addi_i:
		jit_addi_i(lit->r0, lit->r1, lit->i0.i);
		break;
	    case elit_addi_l:
		jit_addi_l(lit->r0, lit->r1, lit->i0.l);
		break;
	    case elit_addi_p:
		jit_addi_p(lit->r0, lit->r1, lit->i0.i);
		break;
	    case elit_addcr_ui:
		jit_addcr_ui(lit->r0, lit->r1, lit->r2);
		break;
	    case elit_addxr_ui:
		jit_addxr_ui(lit->r0, lit->r1, lit->r2);
		break;
	    case elit_addci_ui:
		jit_addci_ui(lit->r0, lit->r1, lit->i0.i);
		break;
	    case elit_addxi_ui:
		jit_addxi_ui(lit->r0, lit->r1, lit->i0.i);
		break;
	    case elit_subr_i:
		jit_subr_i(lit->r0, lit->r1, lit->r2);
		break;
	    case elit_subr_ui:
		jit_subr_ui(lit->r0, lit->r1, lit->r2);
		break;
	    case elit_subr_l:
		jit_subr_l(lit->r0, lit->r1, lit->r2);
		break;
	    case elit_subr_f:
		jit_subr_f(lit->r0, lit->r1, lit->r2);
		break;
	    case elit_subr_d:
		jit_subr_d(lit->r0, lit->r1, lit->r2);
		break;
	    case elit_subi_i:
		jit_subi_i(lit->r0, lit->r1, lit->i0.i);
		break;
	    case elit_subi_l:
		jit_subi_l(lit->r0, lit->r1, lit->i0.l);
		break;
	    case elit_subi_p:
		jit_subi_p(lit->r0, lit->r1, lit->i0.i);
		break;
	    case elit_subcr_ui:
		jit_subcr_ui(lit->r0, lit->r1, lit->r2);
		break;
	    case elit_subxr_ui:
		jit_subxr_ui(lit->r0, lit->r1, lit->r2);
		break;
	    case elit_subci_ui:
		jit_subci_ui(lit->r0, lit->r1, lit->i0.i);
		break;
	    case elit_subxi_ui:
		jit_subxi_ui(lit->r0, lit->r1, lit->i0.i);
		break;
	    case elit_mulr_f:
		jit_mulr_f(lit->r0, lit->r1, lit->r2);
		break;
	    case elit_mulr_d:
		jit_mulr_d(lit->r0, lit->r1, lit->r2);
		break;
	    case elit_divr_f:
		jit_divr_f(lit->r0, lit->r1, lit->r2);
		break;
	    case elit_divr_d:
		jit_divr_d(lit->r0, lit->r1, lit->r2);
		break;
	    case elit_rshi_i:
		jit_rshi_i(lit->r0, lit->r1, lit->i0.i);
		break;
	    case elit_ldi_p:
		jit_ldi_p(lit->r0, lit->i0.p);
		break;
	    case elit_ldxi_c:
		jit_ldxi_c(lit->r0, lit->r1, lit->i0.i);
		break;
	    case elit_ldxi_uc:
		jit_ldxi_uc(lit->r0, lit->r1, lit->i0.i);
		break;
	    case elit_ldxi_s:
		jit_ldxi_s(lit->r0, lit->r1, lit->i0.i);
		break;
	    case elit_ldxi_us:
		jit_ldxi_us(lit->r0, lit->r1, lit->i0.i);
		break;
	    case elit_ldxi_i:
		jit_ldxi_i(lit->r0, lit->r1, lit->i0.i);
		break;
	    case elit_ldxi_ui:
		jit_ldxi_ui(lit->r0, lit->r1, lit->i0.i);
		break;
	    case elit_ldxi_l:
		jit_ldxi_l(lit->r0, lit->r1, lit->i0.i);
		break;
	    case elit_ldxi_ul:
		jit_ldxi_ul(lit->r0, lit->r1, lit->i0.i);
		break;
	    case elit_ldxi_p:
		jit_ldxi_p(lit->r0, lit->r1, lit->i0.i);
		break;
	    case elit_ldxi_f:
		jit_ldxi_f(lit->r0, lit->r1, lit->i0.i);
		break;
	    case elit_ldxi_d:
		jit_ldxi_d(lit->r0, lit->r1, lit->i0.i);
		break;
	    case elit_stxi_c:
		jit_stxi_c(lit->i0.i, lit->r0, lit->r1);
		break;
	    case elit_stxi_s:
		jit_stxi_s(lit->i0.i, lit->r0, lit->r1);
		break;
	    case elit_stxi_i:
		jit_stxi_i(lit->i0.i, lit->r0, lit->r1);
		break;
	    case elit_stxi_l:
		jit_stxi_l(lit->i0.i, lit->r0, lit->r1);
		break;
	    case elit_stxi_p:
		jit_stxi_p(lit->i0.i, lit->r0, lit->r1);
		break;
	    case elit_stxi_f:
		jit_stxi_f(lit->i0.i, lit->r0, lit->r1);
		break;
	    case elit_stxi_d:
		jit_stxi_d(lit->i0.i, lit->r0, lit->r1);
		break;
	    case elit_ner_i:
		jit_ner_i(lit->r0, lit->r1, lit->r2);
		break;
	    case elit_ner_l:
		jit_ner_l(lit->r0, lit->r1, lit->r2);
		break;
	    case elit_ner_d:
		jit_ner_d(lit->r0, lit->r1, lit->r2);
		break;
	    case elit_nei_i:
		jit_nei_i(lit->r0, lit->r1, lit->i0.i);
		break;
	    case elit_nei_l:
		jit_nei_l(lit->r0, lit->r1, lit->i0.l);
		break;
	    case elit_ltr_i:
		jit_ltr_i(lit->r0, lit->r1, lit->r2);
		break;
	    case elit_ltr_ui:
		jit_ltr_ui(lit->r0, lit->r1, lit->r2);
		break;
	    case elit_ltr_l:
		jit_ltr_l(lit->r0, lit->r1, lit->r2);
		break;
	    case elit_ltr_d:
		jit_ltr_d(lit->r0, lit->r1, lit->r2);
		break;
	    case elit_lti_i:
		jit_lti_i(lit->r0, lit->r1, lit->i0.i);
		break;
	    case elit_lti_l:
		jit_lti_l(lit->r0, lit->r1, lit->i0.l);
		break;
	    case elit_ler_i:
		jit_ler_i(lit->r0, lit->r1, lit->r2);
		break;
	    case elit_ler_ui:
		jit_ler_ui(lit->r0, lit->r1, lit->r2);
		break;
	    case elit_ler_l:
		jit_ler_l(lit->r0, lit->r1, lit->r2);
		break;
	    case elit_ler_d:
		jit_ler_d(lit->r0, lit->r1, lit->r2);
		break;
	    case elit_lei_i:
		jit_lei_i(lit->r0, lit->r1, lit->i0.i);
		break;
	    case elit_lei_l:
		jit_lei_l(lit->r0, lit->r1, lit->i0.l);
		break;
	    case elit_eqr_i:
		jit_eqr_i(lit->r0, lit->r1, lit->r2);
		break;
	    case elit_eqr_l:
		jit_eqr_l(lit->r0, lit->r1, lit->r2);
		break;
	    case elit_eqr_d:
		jit_eqr_d(lit->r0, lit->r1, lit->r2);
		break;
	    case elit_eqi_i:
		jit_eqi_i(lit->r0, lit->r1, lit->i0.i);
		break;
	    case elit_eqi_l:
		jit_eqi_l(lit->r0, lit->r1, lit->i0.l);
		break;
	    case elit_ger_i:
		jit_ger_i(lit->r0, lit->r1, lit->r2);
		break;
	    case elit_ger_ui:
		jit_ger_ui(lit->r0, lit->r1, lit->r2);
		break;
	    case elit_ger_l:
		jit_ger_l(lit->r0, lit->r1, lit->r2);
		break;
	    case elit_ger_d:
		jit_ger_d(lit->r0, lit->r1, lit->r2);
		break;
	    case elit_gei_i:
		jit_gei_i(lit->r0, lit->r1, lit->i0.i);
		break;
	    case elit_gei_l:
		jit_gei_l(lit->r0, lit->r1, lit->i0.l);
		break;
	    case elit_gtr_i:
		jit_gtr_i(lit->r0, lit->r1, lit->r2);
		break;
	    case elit_gtr_ui:
		jit_gtr_ui(lit->r0, lit->r1, lit->r2);
		break;
	    case elit_gtr_l:
		jit_gtr_l(lit->r0, lit->r1, lit->r2);
		break;
	    case elit_gtr_d:
		jit_gtr_d(lit->r0, lit->r1, lit->r2);
		break;
	    case elit_gti_i:
		jit_gti_i(lit->r0, lit->r1, lit->i0.i);
		break;
	    case elit_gti_l:
		jit_gti_l(lit->r0, lit->r1, lit->i0.l);
		break;
	    case elit_blti_i:
		lit->i0.p = jit_blti_i(jit_forward(), lit->r0, lit->i1.i);
		break;
	    case elit_blei_i:
		lit->i0.p = jit_blei_i(jit_forward(), lit->r0, lit->i1.i);
		break;
	    case elit_beqi_i:
		lit->i0.p = jit_beqi_i(jit_forward(), lit->r0, lit->i1.i);
		break;
	    case elit_bgei_i:
		lit->i0.p = jit_bgei_i(jit_forward(), lit->r0, lit->i1.i);
		break;
	    case elit_bgti_i:
		lit->i0.p = jit_bgti_i(jit_forward(), lit->r0, lit->i1.i);
		break;
	    case elit_bnei_i:
		lit->i0.p = jit_bnei_i(jit_forward(), lit->r0, lit->i1.i);
		break;
	    case elit_blti_l:
		lit->i0.p = jit_blti_l(jit_forward(), lit->r0, lit->i1.l);
		break;
	    case elit_blei_l:
		lit->i0.p = jit_blei_l(jit_forward(), lit->r0, lit->i1.l);
		break;
	    case elit_beqi_l:
		lit->i0.p = jit_beqi_l(jit_forward(), lit->r0, lit->i1.l);
		break;
	    case elit_bgei_l:
		lit->i0.p = jit_bgei_l(jit_forward(), lit->r0, lit->i1.l);
		break;
	    case elit_bgti_l:
		lit->i0.p = jit_bgti_l(jit_forward(), lit->r0, lit->i1.l);
		break;
	    case elit_bnei_l:
		lit->i0.p = jit_bnei_l(jit_forward(), lit->r0, lit->i1.l);
		break;
	    case elit_boaddr_i:
		lit->i0.p = jit_boaddr_i(jit_forward(), lit->r0, lit->r1);
		break;
	    case elit_boaddr_l:
		lit->i0.p = jit_boaddr_l(jit_forward(), lit->r0, lit->r1);
		break;
	    case elit_boaddi_i:
		lit->i0.p = jit_boaddi_i(jit_forward(), lit->r0, lit->i1.i);
		break;
	    case elit_boaddi_ui:
		lit->i0.p = jit_boaddi_ui(jit_forward(), lit->r0, lit->i1.i);
		break;
	    case elit_boaddi_l:
		lit->i0.p = jit_boaddi_l(jit_forward(), lit->r0, lit->i1.l);
		break;
	    case elit_boaddi_ul:
		lit->i0.p = jit_boaddi_ul(jit_forward(), lit->r0, lit->i1.l);
		break;
	    case elit_bosubr_i:
		lit->i0.p = jit_bosubr_i(jit_forward(), lit->r0, lit->r1);
		break;
	    case elit_bosubr_l:
		lit->i0.p = jit_bosubr_l(jit_forward(), lit->r0, lit->r1);
		break;
	    case elit_bosubi_i:
		lit->i0.p = jit_bosubi_i(jit_forward(), lit->r0, lit->i1.i);
		break;
	    case elit_bosubi_ui:
		lit->i0.p = jit_bosubi_ui(jit_forward(), lit->r0, lit->i1.i);
		break;
	    case elit_bosubi_l:
		lit->i0.p = jit_bosubi_l(jit_forward(), lit->r0, lit->i1.l);
		break;
	    case elit_bger_f:
		lit->i0.p = jit_bger_f(jit_forward(), lit->r0, lit->r1);
		break;
	    case elit_bger_d:
		lit->i0.p = jit_bger_d(jit_forward(), lit->r0, lit->r1);
		break;
	    case elit_jmpi:
		lit->i0.p = jit_jmpi(jit_forward());
		break;
	    case elit_calli:
		jit_calli(lit->i0.p);
		break;
	    case elit_prepare:
		jit_prepare(lit->i0.i);
		break;
	    case elit_pusharg_i:
		jit_pusharg_i(lit->r0);
		break;
	    case elit_pusharg_l:
		jit_pusharg_l(lit->r0);
		break;
	    case elit_pusharg_p:
		jit_pusharg_p(lit->r0);
		break;
	    case elit_finish:
		jit_finish(lit->i0.p);
		break;
	    default:
		abort();
	}
    }
}

static void
flush_code(void)
{
    flush_list(lit_base[icode]);
    flush_list(lit_base[fcode]);
    flush_list(lit_base[acode]);
    lit_base[icode] = lit_tail[icode] = null;
    lit_base[fcode] = lit_tail[fcode] = null;
    lit_base[acode] = lit_tail[acode] = null;
}

static void
push_type(eint32_t type)
{
    if (stack_vector->offset >= stack_vector->length) {
	erenew_vector(stack_vector, stack_vector->length + 8);
	lit_stack = stack_vector->ptr;
    }
    lit_stack[stack_vector->offset++] = type;
}

static eint32_t
pop_type(void)
{
    assert(stack_vector->offset > 0);
    return (lit_stack[--stack_vector->offset]);
}

static eint32_t
top_type(void)
{
    if (stack_vector->offset > 0)
	return (lit_stack[stack_vector->offset - 1]);
    return (t_any);
}

static eint32_t
stack_adjust(eint32_t count)
{
    if (count < 0) {
	if (-count >= stack_vector->offset) {
	    stack_vector->offset = 0;
	    return (t_any);
	}
	stack_vector->offset += count;
    }
    else {
	for (; count; --count)
	    push_type(t_any);
    }
    return (stack_vector->offset > 0 ?
	    lit_stack[stack_vector->offset - 1] : t_any);
}

static void
zero_sign_extend(eint32_t code, eint32_t type)
{
#if __WORDSIZE == 32
    switch (type) {
	case t_int8:
	    new_rshi_i(code, JIT_V2, JIT_V1, 7);
	    break;
	case t_int16:
	    new_rshi_i(code, JIT_V2, JIT_V1, 15);
	    break;
	case t_int32:
	    new_rshi_i(code, JIT_V2, JIT_V1, 31);
	    break;
	case t_uint8:			case t_uint16:
	case t_uint32:
	    new_movi_i(code, JIT_V2, 0);
	    break;
	case t_int:			case t_uint:
	    break;
	default:
	    abort();
    }
#endif
}

static void
float_extend(eint32_t code, eint32_t type)
{
    switch (type) {
	case t_float32:
	    new_extr_f_d(code, JIT_FPR0, JIT_FPR1);
	    break;
	case t_float:
	    new_extr_d_f(code, JIT_FPR1, JIT_FPR0);
	    break;
	default:
	    abort();
    }
}

static void
load_ireg(eint32_t code, jit_gpr_t base, eint32_t offset)
{
#if __WORDSIZE == 32
    new_ldxi_i(code, JIT_V1, base, offset + offsetof(edata_t, s.l));
    new_ldxi_i(code, JIT_V2, base, offset + offsetof(edata_t, s.h));
#else
    new_ldxi_l(code, JIT_V1, base, offset);
#endif
}

static void
load_ireg_type(eint32_t code, eint32_t offset)
{
    switch (lit_type) {
	case t_int8:
	    new_ldxi_c(code, JIT_V1, JIT_R1, offset);
	    break;
	case t_uint8:
	    new_ldxi_uc(code, JIT_V1, JIT_R1, offset);
	    break;
	case t_int16:
	    new_ldxi_s(code, JIT_V1, JIT_R1, offset);
	    break;
	case t_uint16:
	    new_ldxi_us(code, JIT_V1, JIT_R1, offset);
	    break;
	case t_int32:
#if __WORDSIZE == 32
	case t_uint32:
#endif
	    new_ldxi_i(code, JIT_V1, JIT_R1, offset);
	    break;
#if __WORDSIZE == 64
	case t_uint32:
	    new_ldxi_ui(code, JIT_V1, JIT_R1, offset);
	    break;
#endif
	default:
	    load_ireg(code, JIT_R1, offset);
	    break;
    }
}

static void
load_ireg_implicit(eint32_t code)
{
    load_ireg(code, JIT_V0, offsetof(ethread_t, r0) + offsetof(evalue_t, v.i));
}

static void
store_ireg(eint32_t code, jit_gpr_t base, eint32_t offset)
{
#if __WORDSIZE == 32
    new_stxi_i(code, offset + offsetof(edata_t, s.l), base, JIT_V1);
    new_stxi_i(code, offset + offsetof(edata_t, s.h), base, JIT_V2);
#else
    new_stxi_l(code, offset, base, JIT_V1);
#endif
}

static void
store_ireg_value_type(eint32_t code, jit_gpr_t base, eint32_t offset)
{
    new_movi_i(code, JIT_R0, t_int);
    new_stxi_i(code, offset + offsetof(evalue_t, t), base, JIT_R0);
    store_ireg(code, base, offset + offsetof(evalue_t, v.i));
}

static void
store_ireg_implicit(eint32_t code, ebool_t type)
{
    if (type)
	store_ireg_value_type(code, JIT_V0, offsetof(ethread_t, r0));
    else
	store_ireg(code, JIT_V0,
		   offsetof(ethread_t, r0) + offsetof(evalue_t, v.i));
}

#if __WORDSIZE == 32
static void
store_ireg_implicit_rgz(eint32_t code, ebool_t sign, jit_gpr_t l, jit_gpr_t h)
{
    new_prepare(code, 3);
    new_pusharg_i(code, h);
    new_pusharg_i(code, l);
    new_movi_i(code, JIT_R0, t_rgz);
    new_stxi_i(code, offsetof(ethread_t, r0) + offsetof(evalue_t, t),
	       JIT_V0, JIT_R0);
    new_addi_p(code, JIT_V1, JIT_V0,
	       offsetof(ethread_t, qq) + offsetof(__cqq_struct, re) +
	       offsetof(__mpq_struct, _mp_num));
    new_stxi_p(code, offsetof(ethread_t, r0) + offsetof(evalue_t, v.o),
	       JIT_V0, JIT_V1);
    new_pusharg_p(code, JIT_V1);
    if (sign)
	new_finish(code, empz_set_ii);
    else
	new_finish(code, empz_set_uu);
}
#else
static void
store_ireg_implicit_rgz(eint32_t code, ebool_t sign, jit_gpr_t r)
{
    new_prepare(code, 2);
    new_pusharg_l(code, r);
    new_movi_i(code, JIT_R0, t_rgz);
    new_stxi_i(code, offsetof(ethread_t, r0) + offsetof(evalue_t, t),
	       JIT_V0, JIT_R0);
    new_addi_p(code, JIT_V1, JIT_V0,
	       offsetof(ethread_t, qq) + offsetof(__cqq_struct, re) +
	       offsetof(__mpq_struct, _mp_num));
    new_stxi_p(code, offsetof(ethread_t, r0) + offsetof(evalue_t, v.o),
	       JIT_V0, JIT_V1);
    new_pusharg_p(code, JIT_V1);
    if (sign)
	new_finish(code, mpz_set_si);
    else
	new_finish(code, mpz_set_ui);
}
#endif

static void
store_int32(eint32_t code, eint32_t offset, eint32_t type)
{
    switch (type) {
	case t_int8:			case t_uint8:
	    new_stxi_c(code, offset, JIT_R1, JIT_V1);
	    break;
	case t_int16:			case t_uint16:
	    new_stxi_s(code, offset, JIT_R1, JIT_V1);
	    break;
	case t_int32:			case t_uint32:
	    new_stxi_i(code, offset, JIT_R1, JIT_V1);
	    break;
	case t_int:			case t_uint:
	    if (!(lit_sync & lit_sync_int_ext)) {
		zero_sign_extend(code, lit_type);
		lit_sync |= lit_sync_int_ext;
	    }
	    store_ireg(code, JIT_R1, offset);
	    break;
	case t_float32:
	    if (!(lit_sync & lit_sync_int_float)) {
		new_extr_i_f(code, JIT_FPR1, JIT_V1);
		lit_sync = (lit_sync|lit_sync_int_float)&~lit_sync_long_float;
	    }
	    new_stxi_f(code, offset, JIT_R1, JIT_FPR1);
	    break;
	case t_float:
	    if (!(lit_sync & lit_sync_int_double)) {
		new_extr_i_d(code, JIT_FPR0, JIT_V1);
		lit_sync = (lit_sync|lit_sync_int_double)&~lit_sync_long_double;
	    }
	    new_stxi_d(code, offset, JIT_R1, JIT_FPR0);
	    break;
	default:
	    abort();
    }
}

static void
store_int(eint32_t code, eint32_t offset, eint32_t type)
{
    elit_node_t		*done;

    switch (type) {
	case t_int8:			case t_uint8:
	    new_stxi_c(code, offset, JIT_R1, JIT_V1);
	    break;
	case t_int16:			case t_uint16:
	    new_stxi_s(code, offset, JIT_R1, JIT_V1);
	    break;
	case t_int32:			case t_uint32:
	    new_stxi_i(code, offset, JIT_R1, JIT_V1);
	    break;
	case t_int:			case t_uint:
	    store_ireg(code, JIT_R1, offset);
	    break;
	case t_float32:
#if __WORDSIZE == 64
	    if (!(lit_sync & lit_sync_long_float)) {
		new_extr_l_f(code, JIT_FPR1, JIT_V1);
		lit_sync = (lit_sync|lit_sync_long_float)&~lit_sync_int_float;
	    }
	    new_stxi_f(code, offset, JIT_R1, JIT_FPR1);
#else
	    if (!(lit_sync & lit_sync_implicit)) {
		store_ireg_implicit(code, true);
		lit_sync |= lit_sync_implicit;
	    }
	    if (offset)
		new_addi_p(code, JIT_R1, JIT_R1, offset);
	    new_prepare(code, 1);
	    new_pusharg_p(code, JIT_R1);
	    new_finish(code, evm_store_float32);
	    lit_sync &= ~(lit_sync_float_ext|
			  lit_sync_int_float|lit_sync_int_double|
			  lit_sync_long_float|lit_sync_long_double);
#endif
	    break;
	case t_float:
#if __WORDSIZE == 64
	    if (!(lit_sync & lit_sync_long_double)) {
		new_extr_l_d(code, JIT_FPR0, JIT_V1);
		lit_sync = (lit_sync|lit_sync_long_double)&~lit_sync_int_float;
	    }
	    new_stxi_d(code, offset, JIT_R1, JIT_FPR0);
#else
	    if (!(lit_sync & lit_sync_implicit)) {
		store_ireg_implicit(code, true);
		lit_sync |= lit_sync_implicit;
	    }
	    if (offset)
		new_addi_p(code, JIT_R1, JIT_R1, offset);
	    new_prepare(code, 1);
	    new_pusharg_p(code, JIT_R1);
	    new_finish(code, evm_store_float64);
	    lit_sync &= ~(lit_sync_float_ext|
			  lit_sync_int_float|lit_sync_int_double|
			  lit_sync_long_float|lit_sync_long_double);
#endif
	    break;
	default:
	    abort();
    }
    if (lit_type == t_any) {
	assert(code == icode);
	done = new_jmpi(code, null);
	new_label(acode, done);
    }
}

static void
push_ireg(eint32_t code)
{
    new_ldxi_p(code, JIT_R1, JIT_V0, offsetof(ethread_t, esp));
    new_addi_p(code, JIT_R0, JIT_R1, sizeof(evalue_t));
    new_stxi_p(code, offsetof(ethread_t, esp), JIT_V0, JIT_R0);
    new_movi_i(code, JIT_R0, t_int);
    new_stxi_i(code, offsetof(evalue_t, t), JIT_R1, JIT_R0);
    store_ireg(code, JIT_R1, offsetof(evalue_t, v.i));
}

static void
load_freg(eint32_t code, jit_gpr_t base, eint32_t offset)
{
    new_ldxi_d(code, JIT_FPR0, base, offset);
}

static void
load_freg_type(eint32_t code, eint32_t offset)
{
    switch (lit_type) {
	case t_float32:
	    new_ldxi_f(code, JIT_FPR1, JIT_R1, offset);
	    break;
	default:
	    load_freg(code, JIT_R1, offset);
	    break;
    }
}

static void
load_freg_implicit(eint32_t code)
{
    load_freg(code, JIT_V0, offsetof(ethread_t, r0) + offsetof(evalue_t, v.d));
}

static void
store_freg(eint32_t code, jit_gpr_t base, eint32_t offset)
{
    new_stxi_d(code, offset, base, JIT_FPR0);
}

static void
store_freg_value_type(eint32_t code, jit_gpr_t base, eint32_t offset)
{
    new_movi_i(code, JIT_R0, t_float);
    new_stxi_i(code, offset + offsetof(evalue_t, t), base, JIT_R0);
    store_freg(code, base, offset + offsetof(evalue_t, v.d));
}

static void
store_freg_implicit(eint32_t code, ebool_t type)
{
    if (type)
	store_freg_value_type(code, JIT_V0, offsetof(ethread_t, r0));
    else
	store_freg(code, JIT_V0,
		   offsetof(ethread_t, r0) + offsetof(evalue_t, v.d));
}

static void
store_float32(eint32_t code, eint32_t offset, eint32_t type)
{
    switch (type) {
	case t_int8:			case t_uint8:
	    if (!(lit_sync & lit_sync_int_float)) {
		new_truncr_f_i(code, JIT_V1, JIT_FPR1);
		lit_sync = (lit_sync|lit_sync_int_float)&~lit_sync_long_float;
	    }
	    new_stxi_c(code, offset, JIT_R1, JIT_V1);
	    break;
	case t_int16:			case t_uint16:
	    if (!(lit_sync & lit_sync_int_float)) {
		new_truncr_f_i(code, JIT_V1, JIT_FPR1);
		lit_sync = (lit_sync|lit_sync_int_float)&~lit_sync_long_float;
	    }
	    new_stxi_s(code, offset, JIT_R1, JIT_V1);
	    break;
	case t_int32:			case t_uint32:
	    if (!(lit_sync & lit_sync_int_float)) {
		new_truncr_f_i(code, JIT_V1, JIT_FPR1);
		lit_sync = (lit_sync|lit_sync_int_float)&~lit_sync_long_float;
	    }
	    new_stxi_i(code, offset, JIT_R1, JIT_V1);
	    break;
	case t_int:		case t_uint:
#if __WORDSIZE == 64
	    if (!(lit_sync & lit_sync_long_float)) {
		new_truncr_f_l(code, JIT_V1, JIT_FPR1);
		lit_sync = (lit_sync|lit_sync_long_float)&~lit_sync_int_float;
	    }
	    new_stxi_l(code, offset, JIT_R1, JIT_V1);
#else
	    if (!(lit_sync & lit_sync_float_ext))
		float_extend(code, lit_type);
	    store_freg_implicit(code, true);
	    if (offset)
		new_addi_p(code, JIT_R1, JIT_R1, offset);
	    new_prepare(code, 1);
	    new_pusharg_p(code, JIT_R1);
	    new_finish(code, evm_store_int64);
	    load_freg_implicit(code);
	    lit_type = t_float;
#endif
	    break;
	case t_float32:
	    new_stxi_f(code, offset, JIT_R1, JIT_FPR1);
	    break;
	case t_float:
	    if (!(lit_sync & lit_sync_float_ext)) {
		new_extr_f_d(code, JIT_FPR0, JIT_FPR1);
		lit_sync |= lit_sync_float_ext;
	    }
	    store_freg(code, JIT_R1, offset);
	    break;
	default:
	    abort();
    }
}

static void
store_float(eint32_t code, eint32_t offset, eint32_t type)
{
    elit_node_t		*done;

    switch (type) {
	case t_int8:			case t_uint8:
	    if (!(lit_sync & lit_sync_int_double)) {
		new_truncr_d_i(code, JIT_V1, JIT_FPR0);
		lit_sync = (lit_sync|lit_sync_int_double)&~lit_sync_long_double;
	    }
	    new_stxi_c(code, offset, JIT_R1, JIT_V1);
	    break;
	case t_int16:			case t_uint16:
	    if (!(lit_sync & lit_sync_int_double)) {
		new_truncr_d_i(code, JIT_V1, JIT_FPR0);
		lit_sync = (lit_sync|lit_sync_int_double)&~lit_sync_long_double;
	    }
	    new_stxi_s(code, offset, JIT_R1, JIT_V1);
	    break;
	case t_int32:			case t_uint32:
	    if (!(lit_sync & lit_sync_int_double)) {
		new_truncr_d_i(code, JIT_V1, JIT_FPR0);
		lit_sync = (lit_sync|lit_sync_int_double)&~lit_sync_long_double;
	    }
	    new_stxi_i(code, offset, JIT_R1, JIT_V1);
	    break;
	case t_int:			case t_uint:
#if __WORDSIZE == 64
	    if (!(lit_sync & lit_sync_long_double)) {
		new_truncr_d_l(code, JIT_V1, JIT_FPR0);
		lit_sync = (lit_sync|lit_sync_long_double)&~lit_sync_int_double;
	    }
	    new_stxi_l(code, offset, JIT_R1, JIT_V1);
#else
	    if (!(lit_sync & lit_sync_implicit))
		store_freg_implicit(code, true);
	    if (offset)
		new_addi_p(code, JIT_R1, JIT_R1, offset);
	    new_prepare(code, 1);
	    new_pusharg_p(code, JIT_R1);
	    new_finish(code, evm_store_int64);
	    load_freg_implicit(code);
#endif
	    break;
	case t_float32:
	    if (!(lit_sync & lit_sync_float_ext)) {
		new_extr_d_f(code, JIT_FPR1, JIT_FPR0);
		lit_sync |= lit_sync_float_ext;
	    }
	    new_stxi_f(code, offset, JIT_R1, JIT_FPR1);
	    break;
	case t_float:
	    store_freg(code, JIT_R1, offset);
	    break;
	default:
	    abort();
    }
    if (lit_type == t_any) {
	assert(code == fcode);
	done = new_jmpi(code, null);
	new_label(acode, done);
    }
}

static void
push_freg(eint32_t code)
{
    new_ldxi_p(code, JIT_R1, JIT_V0, offsetof(ethread_t, esp));
    new_addi_p(code, JIT_R0, JIT_R1, sizeof(evalue_t));
    new_stxi_p(code, offsetof(ethread_t, esp), JIT_V0, JIT_R0);
    new_movi_i(code, JIT_R0, t_float);
    new_stxi_i(code, offsetof(evalue_t, t), JIT_R1, JIT_R0);
    store_freg(code, JIT_R1, offsetof(evalue_t, v.d));
}

static void
store_dynamic(eint32_t code, eint32_t offset, eint32_t type)
{
    if (offset)
	new_addi_p(code, JIT_R1, JIT_R1, offset);
    new_prepare(code, 1);
    new_pusharg_p(code, JIT_R1);
    switch (type) {
	case t_int8:			case t_uint8:
	    new_finish(code, evm_store_int8);
	    break;
	case t_int16:			case t_uint16:
	    new_finish(code, evm_store_int16);
	    break;
	case t_int32:			case t_uint32:
	    new_finish(code, evm_store_int32);
	    break;
	case t_int:			case t_uint:
	    new_finish(code, evm_store_int64);
	    break;
	case t_float32:
	    new_finish(code, evm_store_float32);
	    break;
	case t_float:
	    new_finish(code, evm_store_float64);
	    break;
	default:
	    abort();
    }
}

static void
call_i(eint32_t code, eobject_t function, eint32_t arg0)
{
    new_prepare(code, 1);
    new_movi_i(code, JIT_R0, arg0);
    new_pusharg_i(code, JIT_R0);
    new_finish(code, function);
}

static void
emit_sync_uint(eint32_t code)
{
    elit_node_t		*node;

#if __WORDSIZE == 32
    node = new_blti_i(code, null, JIT_V2, 0);
#else
    node = new_blti_l(code, null, JIT_V1, 0);
#endif
    if (!(lit_sync & lit_sync_implicit))
	store_ireg_implicit(code, true);
    new_label(code, node);
#if __WORDSIZE == 32
    store_ireg_implicit_rgz(code, false, JIT_V1, JIT_V2);
#else
    store_ireg_implicit_rgz(code, false, JIT_V1);
#endif
    node = new_jmpi(code, null);
    new_label(code, node);
}

/* flush code before returning to parser */
static void
parse_flush(eint32_t count)
{
    flush_code();
    if (count)
	stack_adjust(count);
}

/* reset state */
static void
emit_clear(eint32_t count)
{
    flush_code();
    stack_adjust(count);
    lit_sync = lit_sync_implicit;
    lit_type = t_any;
}

/* flush code before calling extern function that changes implicit */
static void
emit_flush(eint32_t count)
{
    switch (lit_type) {
	case t_int8:			case t_uint8:
	case t_int16:			case t_uint16:
	case t_int32:			case t_uint32:
	    if (!(lit_sync & lit_sync_implicit)) {
		if (!(lit_sync & lit_sync_int_ext))
		    zero_sign_extend(icode, lit_type);
		store_ireg_implicit(icode, true);
	    }
	    break;
	case t_int:
	    if (!(lit_sync & lit_sync_implicit))
		store_ireg_implicit(icode, true);
	    break;
	case t_uint:
	    emit_sync_uint(icode);
	    break;
	case t_float32:
	    if (!(lit_sync & lit_sync_implicit)) {
		if (!(lit_sync & lit_sync_float_ext))
		    float_extend(fcode, lit_type);
		store_freg_implicit(fcode, true);
	    }
	    break;
	case t_float:
	    if (!(lit_sync & lit_sync_implicit))
		store_freg_implicit(fcode, true);
	    break;
	default:
	    break;
    }
    emit_clear(count);
}

/* sync code before calling extern function */
static void
emit_sync(eint32_t count)
{
    switch (lit_type) {
	case t_int8:			case t_uint8:
	case t_int16:			case t_uint16:
	case t_int32:			case t_uint32:
	    if (!(lit_sync & lit_sync_implicit)) {
		if (!(lit_sync & lit_sync_int_ext))
		    zero_sign_extend(icode, lit_type);
		store_ireg_implicit(icode, true);
		lit_sync |= lit_sync_implicit|lit_sync_int_ext;
	    }
	    break;
	case t_int:
	    if (!(lit_sync & lit_sync_implicit)) {
		store_ireg_implicit(icode, true);
		lit_sync |= lit_sync_implicit;
	    }
	    break;
	case t_uint:
	    emit_sync_uint(icode);
	    lit_sync = lit_sync_implicit;
	    lit_type = t_any;
	    break;
	case t_float32:
	    if (!(lit_sync & lit_sync_implicit)) {
		if (!(lit_sync & lit_sync_float_ext))
		    float_extend(fcode, lit_type);
		store_freg_implicit(fcode, true);
	    }
	    /* FIXME depends on using callee saved float register */
	    lit_sync = lit_sync_implicit;
	    lit_type = t_any;
	    break;
	case t_float:
	    if (!(lit_sync & lit_sync_implicit))
		store_freg_implicit(fcode, true);
	    /* FIXME depends on using callee saved float register */
	    lit_sync = lit_sync_implicit;
	    lit_type = t_any;
	    break;
	default:
	    break;
    }
    flush_code();
    if (count)
	stack_adjust(count);
}
#endif
